package com.spider.ask120.webmagic.history;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jdiy.core.Rs;

import com.alibaba.fastjson.JSONObject;
import com.spider.ask120.service.ArticleService;
import com.spider.ask120.service.ChannelService;
import com.spider.ask120.webmagic.Base;
import com.spider.utils.DateUtil;
import com.spider.utils.Function;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.selector.Selectable;

/**
 * 健康资讯 频道
 * @author Jxw
 *
 */
public class Zixun extends Base implements PageProcessor{
	static String jkzxUrl = "http://m.120ask.com/news/";
	static String huatiList = "http://m.120ask.com/news/s/\\w+";
	static String huatiPage = "http://m.120ask.com/news/s/\\w+/\\?p=[0-9]+";
	static String viewRegex = "http://m.120ask.com/news/p/[0-9]+";
	
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	
	public Site getSite() { 
		return site;
	}
		
	// process是定制爬虫逻辑的核心接口
	
	public void process(Page page) { 
		String pic_dir = fun.getProValue(NEWS_PIC_FILE_PATH);
		//进入频道页
		if(page.getUrl().regex(jkzxUrl+"$").match()) { 
			//获取到话题页列表URL 
			List<String> huati = page.getHtml().links().regex(huatiList).all(); 
			page.addTargetRequests(huati);
			try {
				for (String url : huati) {
					int i = url.indexOf("/s/");
					String channel = url.substring(i+3, url.length());
					Rs ch = ChannelService.getChannelByStrIdAndPid(channel, NEWS_CHANNEL_PID);
					if (ch.isNull()) {
						Rs cha = new Rs("channel");
						cha.set("str_id", channel);
						cha.set("name", getNameByStr(channel));
						cha.set("parent_id", NEWS_CHANNEL_PID);
						cha.set("level", 2);
						ChannelService.save(cha);
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		//进入话题页
		}else if(page.getUrl().regex(huatiList+"\\?{0}$").match()){
			//遍历话题分页URL，及每页文章  
			String htPage = page.getUrl()+"/?p=1";  
			page.addTargetRequest(htPage);
			
		//进入分页
		}else if(page.getUrl().regex(huatiPage).match()){ 
			try {
				String url = page.getUrl().get();
				int i = url.indexOf("/s/");
				int m = url.lastIndexOf("/");
				String channel = url.substring(i+3, m);
//				System.err.println(channel);
				Rs ch = ChannelService.getChannelByStrIdAndPid(channel,NEWS_CHANNEL_PID);
				List<String> viewPage = page.getHtml().links().regex(viewRegex).all(); 
				for(String view : viewPage){
					//System.out.println(view);
					String id = view.substring(view.indexOf("/p/")+3);
					//System.out.println("id : " + id); 
					String image = page.getHtml().xpath("//a[@href='"+view+"']/dt/img/@src").get();
					//System.out.println("image : " + image); 
					String title = page.getHtml().xpath("//a[@href='"+view+"']/dd/h3/html()").get();
					//System.out.println("title : " + title); 
					String abs = page.getHtml().xpath("//a[@href='"+view+"']/dd[2]/html()").get();
					//System.out.println("abstract : " + abs); 
					
					//存入数据库
					
					if (!ch.isNull()) {
						JSONObject abst = new JSONObject();
						//下载图片到本地并返回图片地址
						String pic_path = fun.downloadPic(image,pic_dir+channel+"/list/");
						abst.put("pic", pic_path);
						abst.put("abst", abs);
						Rs art = ArticleService.getArticleByStrId(ch.get("id"), title, id);
						String a_id = "";
						if (art.isNull()) {
							Rs article = new Rs("article");
							article.set("title", title);
							article.set("str_id", id);
							article.set("abst", abst.toJSONString());
							article.set("channel_id", ch.getInt("id"));
							ArticleService.saveArticle(article);
							a_id = article.get("id");
						}else {
							a_id = art.get("id");
						}
						//System.out.println(view+"?article_id="+a_id);
						page.addTargetRequest(view+"?article_id="+a_id);
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
			
			//判断下一页URL是否存在 
			String path = page.getUrl().toString();
			String pageIndex = path.substring(path.indexOf("p=")+2); 
			String nextPage = path.substring(0, path.indexOf("p=")+2) + (Integer.parseInt(pageIndex)+1);
			if(page.getHtml().links().regex("http://m.120ask.com/news/s/\\w+/\\?p=["+pageIndex+"]").match()){
				//判断当前页 是否有文章列表
				if(page.getHtml().$(".list_box dl").match()){
					page.addTargetRequest(nextPage);
				}
			}
			
			
		//进入详情页
		}else if(page.getUrl().regex(viewRegex).match()){
			//page.putField("title", page.getHtml().xpath("//div[@class='pr']/h2/text()")); 
			Selectable create_time = page.getHtml().$(".s_p span","text");
			//page.putField("create_time", create_time);
			Selectable source = page.getHtml().$(".s_p span:nth-child(2)","text");
			//page.putField("source", source);
			Selectable click = page.getHtml().$("#r_num","text");
			//page.putField("click", click);
			Selectable praise = page.getHtml().$("#g_num","text");
			//page.putField("praise", praise);
			Selectable pageContent = page.getHtml().xpath("//div[@class='s_content']/html()");
			//page.putField("pageContent", pageContent); 
			try {
				String url = page.getUrl().get();
				String id = url.substring(url.indexOf("_id=")+4,url.length());
				Rs art = ArticleService.getArticleById(id);
				if (!art.isNull()) {
					art.set("create_time", create_time.get());
					art.set("source", source.get());
					art.set("browse_count", click.get());
					art.set("praise_count", praise.get());
					String replace = replaceForNews(pageContent.get(),pic_dir
							+"article/"+DateUtil.monthDateT(create_time.get())+"/");
					//System.out.println(pageContent.get());
					//System.out.println(replace);
					art.set("content", replace);
					ArticleService.saveArticle(art);
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
	
	private String getNameByStr(String str){
		Map<String,String> map = new HashMap<String,String>();
		map.put("ylbg", "八卦");
		map.put("nxzt", "女性");
		map.put("clsss", "时尚");
		map.put("nxht", "话题");
		map.put("yszd", "养生");
		map.put("bgpt", "曝光");
		map.put("spaq", "食品安全");
		map.put("lnjb", "老人");
		map.put("hyzn", "怀孕");
		map.put("neike", "内科");
		map.put("waike", "外科");
		map.put("xingbingke", "性病");
		map.put("erke", "儿科");
		map.put("xlcs", "心理");
		map.put("qgkjj", "进入情感空间");
		map.put("lrbj", "老人");
		map.put("lrxw", "进入老人新闻");
		map.put("yaonews", "药界");
		map.put("quanzi", "圈子资讯");
		return map.get(str);
	}
	
	public static void runZixun(String url){
		System.setProperty("http.maxRedirects", "50");  
        System.getProperties().setProperty("proxySet", "true");   
        Function fun = new Function();
		String spider_scheduler_path = fun.getProValue(NEWS_SCHEDULER_PATH);
		FileCacheQueueScheduler scheduler = new FileCacheQueueScheduler(spider_scheduler_path);
		Spider s =Spider.create(new Zixun()).addUrl(url).scheduler(scheduler);
		s.thread(1).start();
	}
	
	public static void main(String[] args) throws Exception {
		System.setProperty("http.maxRedirects", "50");  
        System.getProperties().setProperty("proxySet", "true");   
        Function fun = new Function();
		String spider_scheduler_path = fun.getProValue(NEWS_SCHEDULER_PATH);
		FileCacheQueueScheduler scheduler = new FileCacheQueueScheduler(spider_scheduler_path);
		Spider s =Spider.create(new Zixun()).addUrl("http://m.120ask.com/news/").scheduler(scheduler);
		SpiderMonitor.instance().register(s);
		s.thread(1).start();
    }
 
}
